package com.coderfans.utils;

import org.apache.commons.io.FileUtils;

import java.io.File;
import java.io.IOException;
import java.util.*;

/**
 * Created by ${fanchunshuai} on 2017-7-19.
 *
 * @version: V1.0
 * @Desc:
 * @Copyright (c) 2017 58到家-版权所有
 */
public class ReadCsdnUserLog {
    public static void main(String[] args) throws IOException {
        /*File file = new File("E:\\spiderFile\\csdnSpiderLog6.log");
        try {
            List<String> list = FileUtils.readLines(file,"UTF-8");
            Set<String> set = new HashSet<>();
            for (int i = 0;i<list.size();i++)
                if (list.get(i).length() > 2) {
                    String [] array = list.get(i).replace("[","").replace("]","").split(",");
                    //System.out.println(JSON.toJSONString(array));
                    Collections.addAll(set,array);
                }
            System.out.println(list.size());
            System.out.println("set = "+set.size());
        } catch (IOException e) {
            e.printStackTrace();
        }*/


        File file = new File("E:\\\\spiderFile\\csdnSpiderLog7.log");
        File file2 = new File("E:\\spiderFile\\csdnSpiderLog5.log");
        File file3 = new File("E:\\spiderFile\\csdnSpiderLog6.log");

        File file4 = new File("E:\\spiderFile\\csdnUserLinkInfo.txt");

        List<String> list = FileUtils.readLines(file,"UTF-8");

        List<String> list2 = FileUtils.readLines(file2,"UTF-8");

        List<String> list3 = FileUtils.readLines(file3,"UTF-8");

        List<String> list4 = FileUtils.readLines(file4,"UTF-8");
        System.out.println(list4.size()+"----------------------");

        Set<String> set = new HashSet<>();
        for (int i = 0;i<list.size();i++)
            if (list.get(i).length() > 2) {

                String [] array = list.get(i).split(":");
                if(array.length >=3 ){
                   // System.out.println(array[2]);
                    String userName [] = array[2].split(",");
                    for (String user:userName){
                        Collections.addAll(set,"http://my.csdn.net/"+user);
                    }
                }

                //
            }

        for (int i = 0;i<list2.size();i++) {
            if (list2.get(i).length() > 2) {
                String[] array = list2.get(i).replace("#", "").replace("[", "").replace("]", "").split(",");
                //System.out.println(JSON.toJSONString(array));
                Collections.addAll(set, array);
            }
        }

        for (int i = 0;i<list3.size();i++) {
            if (list3.get(i).length() > 2) {
                String[] array = list3.get(i).replace("#", "").replace("[", "").replace("]", "").split(",");
                //System.out.println(JSON.toJSONString(array));
                Collections.addAll(set, array);
            }
        }
        System.out.println(list.size());
        System.out.println("set = "+set.size());
        String[] arrayx = new String[list4.size()];
        for (int i = 0;i<list4.size();i++) {
            if (list4.get(i).length() > 2) {
                String url  = list3.get(i).replace("\n", "");
                arrayx[i] = url;
            }
        }
        Collections.addAll(set,arrayx);

        System.out.println("set2 = "+set.size());
        File userFile = new File("E:\\spiderFile\\csdnUserLink2.txt");
        Iterator<String> it = set.iterator();
        int i = 0;
        StringBuffer buffer = new StringBuffer();
        while (it.hasNext()){
            i++;
            buffer.append(it.next()+",");
            if(i % 10 == 0){
                buffer.append("\n");
            }
        }
        FileUtils.writeStringToFile(userFile,buffer.toString());

    }
}
